class MonteCarloTrainer {

    constructor() {
        this.countTable = {};
    }

    train(player, env, iter, initState, eps) {
        let lengthList = [];
        let resList = [];
        for (let i = 0; i < iter; i++) {
            env.reset();
            initState && env.setState(Role.Black, initState);
            // 记录状态序列、动作和奖励序列
            let stateList1 = [];
            let actionList1 = [];
            let rewardList1 = [];
            let stateList2 = [];
            let actionList2 = [];
            let rewardList2 = [];
            let count = 0;
            let ret = null;
            let role1 = env.getNextRole();
            let role2 = role1 === Role.Black ? Role.White : Role.Black;
            while (true) {
                count++;
                if (count > 100) {
                    console.error("训练过程未能正常停止。");
                    break;
                }

                stateList1.push(env.getState(role1));
                let action1 = player.getAction(role1, env, eps);
                actionList1.push(action1);
                ret = env.play(role1, action1);
                if (ret.gameOver)
                    break;
                if (rewardList2.length < actionList2.length)
                    rewardList2.push(env.getReward(role2));

                stateList2.push(env.getState(role2));
                let action2 = player.getAction(role2, env, eps);
                actionList2.push(action2);
                ret = env.play(role2, action2);
                if (ret.gameOver)
                    break;
                if (rewardList1.length < actionList1.length) 
                    rewardList1.push(env.getReward(role1));
            }
            if (rewardList1.length < actionList1.length) 
                rewardList1.push(env.getReward(role1));
            if (rewardList2.length < actionList2.length)
                rewardList2.push(env.getReward(role2));
            stateList1.push(env.getState(Role.Black));
            stateList2.push(env.getState(Role.White));
            this.updateQTable(player.qTable, this.countTable, stateList1, actionList1, rewardList1);
            this.updateQTable(player.qTable, this.countTable, stateList2, actionList2, rewardList2);
            // console.log(stateList1);
            // console.log(actionList1);
            // console.log(rewardList1);
            // console.log(stateList2);
            // console.log(actionList2);
            // console.log(rewardList2);
            // console.log(player.qTable);
            lengthList.push(stateList1.length);
            if (rewardList1.length > 0) {
                resList.push(rewardList1[rewardList1.length - 1]);
            }
        }
        // console.log(lengthList);
        // console.log(resList);
        // console.log(this.countTable);
    }

    updateQTable(qTable, countTable, stateList, actionList, rewardList) {
        console.assert(stateList.length === actionList.length + 1);
        console.assert(actionList.length == rewardList.length);
        let rewardSum = 0;
        for (let t = rewardList.length - 1; t >= 0; t--) {
            rewardSum += rewardList[t];
            let rewardMean = rewardSum / (rewardList.length - t);
            let key = qTableKey(stateList[t], actionList[t]);
            let q = qTable[key] || 0;
            let count = countTable[key] || 0;
            qTable[key] = (q * count + rewardMean) / (count + 1);
            countTable[key] = count + 1;
        }
    }
}